package edu.cn.nlsde.tmfst.crawler.handler;

import edu.cn.nlsde.tmfst.crawler.common.DonothingDuplicateRemover;
import edu.cn.nlsde.tmfst.crawler.processor.weiboCrawler;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.scheduler.QueueScheduler;

/**
 * Created by dell on 2016/5/5.
 */
public class weiboHandler {
    public static void crawler_weibo() {
        Spider.create(new weiboCrawler())
                .addUrl("https://api.weibo.com/2/statuses/public_timeline.json?access_token=2.00LvlIACBdqPBE90fe20608bT2HIgD&count=200")
                .setScheduler(new QueueScheduler().setDuplicateRemover(new DonothingDuplicateRemover()))
                .thread(1)
                .run();
    }
}
